{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The instruction bellow automagically reload Optimus if you made any modification on the library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If you modify Optimus or any library this code is going to reload it\n",
    "%load_ext autoreload\n",
    "%autoreload "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If you are in the example folder. This is the way to find optimus\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append(\"..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Optimus\n",
    "from optimus import Optimus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "    You are using PySparkling of version 2.4.10, but your PySpark is of\n",
      "    version 2.3.1. Please make sure Spark and PySparkling versions are compatible. \n"
     ]
    }
   ],
   "source": [
    "op = Optimus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Go to optimus/create.py. In 'def data_frame()' write print(\"Hello World\"). Now run the cell below and you should see \"Hello World\" bellow the cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from pyspark.sql.types import StringType, IntegerType, ArrayType\n",
    "\n",
    "df = op.create.df(\n",
    "    [\n",
    "        (\"words\", \"str\", True),\n",
    "        (\"num\", \"int\", True),\n",
    "        (\"animals\", \"str\", True),\n",
    "        (\"thing\", StringType(), True),\n",
    "        (\"two strings\", StringType(), True),\n",
    "        (\"filter\", StringType(), True),\n",
    "        (\"num 2\", \"string\", True),\n",
    "        (\"col_array\", ArrayType(StringType()), True),\n",
    "        (\"col_int\", ArrayType(IntegerType()), True)\n",
    "\n",
    "    ]\n",
    "    ,\n",
    "    [\n",
    "        (\"  I like     fish  \", 1, \"dog\", \"housé\", \"cat-car\", \"a\", \"1\", [\"baby\", \"sorry\"], [1, 2, 3]),\n",
    "        (\"    zombies\", 2, \"cat\", \"tv\", \"dog-tv\", \"b\", \"2\", [\"baby 1\", \"sorry 1\"], [3, 4]),\n",
    "        (\"simpsons   cat lady\", 2, \"frog\", \"table\", \"eagle-tv-plus\", \"1\", \"3\", [\"baby 2\", \"sorry 2\"], [5, 6, 7]),\n",
    "        (None, 3, \"eagle\", \"glass\", \"lion-pc\", \"c\", \"4\", [\"baby 3\", \"sorry 3\"], [7, 8])\n",
    "    ])\n",
    "\n",
    "# df.ext.display()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "<div class=\"info_items\" style=\"margin: 10px 0; font-family: sans-serif; font-size: 10px;\">Viewing 4 of 4 rows / 9 columns</div>\n",
       "<div class=\"info_items\" style=\"margin: 10px 0; font-family: sans-serif; font-size: 10px;\">8 partition(s)</div>\n",
       "\n",
       "<table class=\"optimus_table\" style=\"font-size: 12px;\">\n",
       "    <thead style=\"border-bottom: 1px solid black;\">\n",
       "    <tr>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">words</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">1 (string)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">num</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">2 (int)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">animals</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">3 (string)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">thing</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">4 (string)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">two strings</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">5 (string)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">filter</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">6 (string)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">num 2</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">7 (string)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">col_array</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">8 (array&lt;string&gt;)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "        <th style=\"font-family: sans-serif;\">\n",
       "            <div class=\"column_name\" style=\"font-size: 1.2em;\">col_int</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">9 (array&lt;int&gt;)</div>\n",
       "            <div class=\"data_type\" style=\"font-size: 0.8em; font-weight: normal;\">\n",
       "                \n",
       "                nullable\n",
       "                \n",
       "            </div>\n",
       "        </th>\n",
       "        \n",
       "    </tr>\n",
       "\n",
       "    </thead>\n",
       "    <tbody style=\"font-family: monospace; border-bottom: 1px solid #cccccc;\">\n",
       "    \n",
       "    <tr>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"&#8901;&#8901;I&#8901;like&#8901;&#8901;&#8901;&#8901;&#8901;fish&#8901;&#8901;\">\n",
       "                \n",
       "                &#8901;&#8901;I&#8901;like&#8901;&#8901;&#8901;&#8901;&#8901;fish&#8901;&#8901;\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"1\">\n",
       "                \n",
       "                1\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"dog\">\n",
       "                \n",
       "                dog\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"housé\">\n",
       "                \n",
       "                housé\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"cat-car\">\n",
       "                \n",
       "                cat-car\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"a\">\n",
       "                \n",
       "                a\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"1\">\n",
       "                \n",
       "                1\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"['baby',&#8901;'sorry']\">\n",
       "                \n",
       "                ['baby',&#8901;'sorry']\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"[1,&#8901;2,&#8901;3]\">\n",
       "                \n",
       "                [1,&#8901;2,&#8901;3]\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "    </tr>\n",
       "    \n",
       "    <tr>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"&#8901;&#8901;&#8901;&#8901;zombies\">\n",
       "                \n",
       "                &#8901;&#8901;&#8901;&#8901;zombies\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"2\">\n",
       "                \n",
       "                2\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"cat\">\n",
       "                \n",
       "                cat\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"tv\">\n",
       "                \n",
       "                tv\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"dog-tv\">\n",
       "                \n",
       "                dog-tv\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"b\">\n",
       "                \n",
       "                b\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"2\">\n",
       "                \n",
       "                2\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"['baby&#8901;1',&#8901;'sorry&#8901;1']\">\n",
       "                \n",
       "                ['baby&#8901;1',&#8901;'sorry&#8901;1']\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"[3,&#8901;4]\">\n",
       "                \n",
       "                [3,&#8901;4]\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "    </tr>\n",
       "    \n",
       "    <tr>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"simpsons&#8901;&#8901;&#8901;cat&#8901;lady\">\n",
       "                \n",
       "                simpsons&#8901;&#8901;&#8901;cat&#8901;lady\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"2\">\n",
       "                \n",
       "                2\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"frog\">\n",
       "                \n",
       "                frog\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"table\">\n",
       "                \n",
       "                table\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"eagle-tv-plus\">\n",
       "                \n",
       "                eagle-tv-plus\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"1\">\n",
       "                \n",
       "                1\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"3\">\n",
       "                \n",
       "                3\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"['baby&#8901;2',&#8901;'sorry&#8901;2']\">\n",
       "                \n",
       "                ['baby&#8901;2',&#8901;'sorry&#8901;2']\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"[5,&#8901;6,&#8901;7]\">\n",
       "                \n",
       "                [5,&#8901;6,&#8901;7]\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "    </tr>\n",
       "    \n",
       "    <tr>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"None\">\n",
       "                \n",
       "                None\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"3\">\n",
       "                \n",
       "                3\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"eagle\">\n",
       "                \n",
       "                eagle\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"glass\">\n",
       "                \n",
       "                glass\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"lion-pc\">\n",
       "                \n",
       "                lion-pc\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"c\">\n",
       "                \n",
       "                c\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"4\">\n",
       "                \n",
       "                4\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"['baby&#8901;3',&#8901;'sorry&#8901;3']\">\n",
       "                \n",
       "                ['baby&#8901;3',&#8901;'sorry&#8901;3']\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "        <td style=\"border: 0px;\">\n",
       "            <div style=\"min-height: 14px;\" title=\"[7,&#8901;8]\">\n",
       "                \n",
       "                [7,&#8901;8]\n",
       "                 \n",
       "            </div>\n",
       "        </td>\n",
       "        \n",
       "    </tr>\n",
       "    \n",
       "    </tbody>\n",
       "</table>\n",
       "\n",
       "\n",
       "<div class=\"info_items\" style=\"margin: 10px 0; font-family: sans-serif; font-size: 10px;\">Viewing 4 of 4 rows / 9 columns</div>\n",
       "<div class=\"info_items\" style=\"margin: 10px 0; font-family: sans-serif; font-size: 10px;\">8 partition(s)</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.ext.display()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
